31. 下一季度 TTM 股息率为正状态识别（K近邻算法）

from pathlib import Path  # 导入路径工具以探测课程数据挂载
import numpy as np  # 导入数值计算库
import pandas as pd  # 导入数据分析库
from sklearn.preprocessing import StandardScaler  # 导入标准化器
from sklearn.neighbors import KNeighborsClassifier  # 导入KNN分类器
from sklearn.model_selection import GridSearchCV  # 导入网格搜索并接收自定义季度前向折
from sklearn.metrics import confusion_matrix, accuracy_score, recall_score, precision_score  # 导入评估指标
from sklearn.pipeline import Pipeline  # 导入管道以便在每个交叉验证训练折内拟合预处理
from sklearn.preprocessing import FunctionTransformer  # 导入无状态变换器以执行标准化后截尾
candidate_roots = [Path('/home/ubuntu/r2_data_mount/data'), Path(r'C:\qiufei\data')]  # 按课程规定顺序探测数据根目录
data_root = next((root for root in candidate_roots if root.is_dir()), None)  # 选择第一个可用根目录
factor_path = data_root / 'stock' / 'valuation_factors_quarterly_15_years.h5' if data_root else None  # 定位估值因子季度表
if factor_path is None or not factor_path.is_file():  # 真实数据缺失时明确提示
    raise FileNotFoundError(f'未找到估值因子文件：{factor_path}；请从课程数据下载入口获取')  # 提示读者下载文件

valuation_panel = pd.read_hdf(factor_path, key='valuation_factors')  # 读取真实A股估值因子面板
feature_columns = ['market_cap', 'pb_ratio_lf', 'ps_ratio_ttm', 'ep_ratio_ttm']  # 规模与估值四特征
dividend_panel = valuation_panel[feature_columns + ['dividend_yield_ttm']].replace([np.inf, -np.inf], np.nan).dropna()  # 去除缺失与无穷值
dividend_panel = dividend_panel.reset_index().sort_values(['order_book_id', 'date'])  # 展开索引并按公司与报告期排序
calendar_quarters = pd.date_range(dividend_panel['date'].min(), dividend_panel['date'].max(), freq='QE')  # 构建完整日历季度网格
company_frames = []  # 收集每公司按日历重索引的结果
for company_code, company_frame in dividend_panel.groupby('order_book_id'):  # 逐公司处理
    reindexed_frame = company_frame.set_index('date').drop(columns='order_book_id').reindex(calendar_quarters)  # 把数据缺口显式补为缺失行
    reindexed_frame['order_book_id'] = company_code  # 回填公司代码
    company_frames.append(reindexed_frame.reset_index().rename(columns={'index': 'date'}))  # 收集结果
dividend_panel = pd.concat(company_frames, ignore_index=True)  # 合并为日历对齐面板
print(f'日历网格：{len(calendar_quarters)}个季度｜覆盖公司：{dividend_panel["order_book_id"].nunique()}家')  # 报告日历对齐输入

next_quarter_yield = dividend_panel.groupby('order_book_id')['dividend_yield_ttm'].shift(-1)  # 同一公司下一日历季度TTM股息率
dividend_panel['next_ttm_yield_positive'] = next_quarter_yield > 0  # 标签：下一季度TTM股息率是否大于0
dividend_panel = dividend_panel[next_quarter_yield.notna()].dropna(subset=feature_columns)  # 仅保留标签与特征完整的行
classroom_panel = dividend_panel[dividend_panel['date'] >= '2019-01-01'].copy()  # 保留2019年起的课堂样本
classroom_panel['log_market_cap'] = np.log(classroom_panel['market_cap'])  # 市值对数化压缩尺度
print(f'输入口径：{len(classroom_panel)}个公司—季度样本（{classroom_panel["date"].min().date()}—{classroom_panel["date"].max().date()}）｜下一季度TTM股息率为正占比{classroom_panel["next_ttm_yield_positive"].mean():.1%}')  # 报告样本与标签分布

model_features = ['log_market_cap', 'pb_ratio_lf', 'ps_ratio_ttm', 'ep_ratio_ttm']  # 模型输入特征
train_mask = classroom_panel['date'] <= '2023-12-31'  # 名义训练期≤2023Q4；因面板季度缺失，有效标签止于2023Q1
test_mask = classroom_panel['date'] >= '2024-01-01'  # 名义测试期≥2024Q1；有效样本外观测自2024Q3起
x_train_raw = classroom_panel.loc[train_mask, model_features]  # 训练特征（标准化前）
x_test_raw = classroom_panel.loc[test_mask, model_features]  # 测试特征（标准化前）
y_train = classroom_panel.loc[train_mask, 'next_ttm_yield_positive']  # 提取训练期TTM股息率状态标签
y_test = classroom_panel.loc[test_mask, 'next_ttm_yield_positive']  # 提取样本外TTM股息率状态标签
majority_baseline = max(y_test.mean(), 1 - y_test.mean())  # 多数类基线准确率
print(f'时间划分：训练{len(y_train)}条（有效至2023Q1）｜测试{len(y_test)}条（自2024Q3）｜多数类基线{majority_baseline:.3f}｜标准化将在每个训练折内拟合')  # 报告有效划分、基线与无泄漏口径

clip_transformer = FunctionTransformer(np.clip, kw_args={'a_min': -5, 'a_max': 5})  # 将标准化后的极端值截到正负5且不估计额外参数
knn_pipeline = Pipeline([('scale', StandardScaler()), ('clip', clip_transformer), ('knn', KNeighborsClassifier())])  # 把标准化、截尾与KNN封装为同一管道
k_param_grid = {'knn__n_neighbors': [3, 5, 7, 9, 11, 13]}  # 使用管道参数名扫描候选K值
training_dates = classroom_panel.loc[train_mask, 'date'].reset_index(drop=True)  # 保存每条训练样本的季度键
candidate_validation_dates = sorted(training_dates.unique())[-4:]  # 取训练期末四个可用季度作前向验证块
forward_splits = []  # 收集严格早期训练到后期验证的索引对
for validation_date in candidate_validation_dates:  # 逐季度建立扩展窗验证折
    fold_train = np.flatnonzero(training_dates.lt(validation_date).to_numpy())  # 只允许更早季度进入折内训练
    fold_valid = np.flatnonzero(training_dates.eq(validation_date).to_numpy())  # 同一季度公司共同进入验证块
    if len(fold_train) and len(fold_valid): forward_splits.append((fold_train, fold_valid))  # 保存有效时间折
if len(forward_splits) < 3: raise ValueError('输入文件、字段、样本量或数值不符合当前分析要求，请按本页说明检查')  # 前向折不足时停止调参
grid_search = GridSearchCV(knn_pipeline, param_grid=k_param_grid, cv=forward_splits, n_jobs=1)  # 每折只用更早季度拟合预处理与KNN
grid_search.fit(x_train_raw.reset_index(drop=True), y_train.reset_index(drop=True))  # 最终时间外测试集不参与调优
print(f'最优K={grid_search.best_params_["knn__n_neighbors"]}｜季度前向{len(forward_splits)}折准确率{grid_search.best_score_:.3f}')  # 展示前向调优结果

y_predict = grid_search.predict(x_test_raw)  # 管道用训练期参数转换并预测样本外TTM股息率状态
confusion_table = confusion_matrix(y_test, y_predict)  # 混淆矩阵（行=实际，列=预测）
test_accuracy = accuracy_score(y_test, y_predict)  # 样本外准确率
positive_status_recall = recall_score(y_test, y_predict)  # 正状态召回率：真实TTM股息率为正中被找出的比例
positive_status_precision = precision_score(y_test, y_predict)  # 正状态精度：预测为正中真实为正的比例
improvement_pp = (test_accuracy - majority_baseline) * 100  # 相对基线的提升幅度（百分点）
margin_limit, recall_limit, precision_limit = 3.0, 0.85, 0.85  # 预声明的三条检查要求阈值
conditions_met = improvement_pp >= margin_limit and positive_status_recall >= recall_limit and positive_status_precision >= precision_limit  # 三条件同时满足才通过
decision = '允许生成课堂TTM状态核对队列' if conditions_met else '检查要求未通过，暂停状态核对队列输出'  # 课堂建议
print(f'混淆矩阵[[TN,FP],[FN,TP]]：{confusion_table.tolist()}')  # 展示分类明细
print(f'样本外准确率{test_accuracy:.3f}（基线{majority_baseline:.3f}，+{improvement_pp:.1f}pp）｜正状态召回{positive_status_recall:.1%}｜正状态精度{positive_status_precision:.1%}')  # 展示验证指标
print(f'检查要求（优于基线≥{margin_limit:.0f}pp、召回≥{recall_limit:.0%}、精度≥{precision_limit:.0%}）：{"通过" if conditions_met else "暂停"}｜{decision}')  # 展示检查要求与条件动作